package com.chance.cc.crawler.development.scripts.xcar.xCarConfig;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.Downloader;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import jdk.nashorn.api.scripting.NashornScriptEngineFactory;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.script.*;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * @ClassName XCarConfigCrawlerScript
 * @Description TODO
 * @Author songding
 * @Date 2021/8/19 14:38
 * @Version 1.0
 *  * 车型配置信息采集
 **/
public class XCarConfigCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(XCarConfigCrawlerScript.class);

    private static final String DOMAIN = "xcar";
    private static final String SIDE = "config";
    private static final String RECORD_AGAIN_REQUEST = "record_again_request";//判断重新下载
    private static final String TURN_PAGE_ITEM_REQUEST = "turn_page_item_request";//判断重新下载
    private static final String KEYS = "keys";


    private Downloader httpDownload;
    private static Object httpDownloadObj = new Object();

    private static final String XcarUrl = "https://newcar.xcar.com.cn";
    private static final String Xcar = "https://newcar.xcar.com.cn/";

    //private static final String
    private static final String Url = "https://newcar.xcar.com.cn/\\S*/";
    private static final String OLDYEAR = "https://newcar.xcar.com.cn/auto/index.php\\?r=newcar/SeriseParentIndex/\\S*";//停售款链接
    private static final String ConfigUrl = "https://newcar.xcar.com.cn/\\S*/config.htm";

    public Map<String, CompiledScript> compiledScriptMap;
    public ScriptEngine scriptEngine;
    @Override
    public String domain() {
        return DOMAIN;
    }
    @Override
    public void initUrlRegulars() {
        addUrlRegular(Xcar);
        addUrlRegular(Url);
        addUrlRegular(OLDYEAR);
        addUrlRegular(ConfigUrl);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        return crawlerRecord.tagsCreator().bizTags().site().equals(SIDE);
    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords){
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords){
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")){
                this.initKeyWord(crawlerRequestRecord,supportSourceRecord,prepareLinks);
            }
        }
        return prepareLinks;
    }


    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord, List<CrawlerRecord> prepareLinks) {
        if (supportSourceRecord == null){return;}
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type+".msg").get();
        if (!"success".equals(msg)){
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all){
            JSONObject jsonObject = JSONObject.parseObject(data);
            String  keyword = (String) jsonObject.get("keyword");
            String url = "https://newcar.xcar.com.cn/"+keyword+"/";
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRequestRecord)
                    .httpUrl(url)
                    .releaseTime(System.currentTimeMillis())
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            prepareLinks.add(record);
        }
    }


    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parseLinks = new ArrayList<>();
        String url = crawlerRecord.getHttpRequest().getUrl();
        if ((!page.isDownloadSuccess()) || page.getStatusCode() != 200) {
            log.error("download page !=200 or page ==[{}]",page.getStatusCode());
            if (page.getStatusCode() == 521) {
                initCompileScript();//初始化加密方式
                this.cookieUpdate(page,crawlerRecord,parseLinks);
            }
            this.recordAgainRequest(crawlerRecord, parseLinks);
            crawlerRecord.setNeedWashPage(false);//下载异常，不需要清洗页面
            return parseLinks;
        }
        if (url.matches(Url)) {
            this.parseBrandMessage(crawlerRecord, page, parseLinks);
            this.parseTurnPage(crawlerRecord, page, parseLinks);
            this.parseConfigUrl(crawlerRecord, page, parseLinks);
        }
        if (url.matches(ConfigUrl)) {
            crawlerRecord.setNeedWashPage(true);
        }
        if (url.matches(OLDYEAR)) {//清洗停售款
            this.parseOld(crawlerRecord, page, parseLinks);
        }
        return parseLinks;
    }

    /*
    * 清洗停售款的url
    * */
    private void parseOld(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"ps_stop_model_list_table\"]/tbody/tr").nodes();
        for (int i = 1; i < nodes.size(); i++) {
            String configUrl = nodes.get(i).xpath("./td[5]/p/a[2]/@href").get();
            if (configUrl==null){
                log.error("There's no row is url " + crawlerRecord.getHttpRequest().getUrl() +"  Take the failure  "+ i+" row");
                continue;
            }
            configUrl = XcarUrl + configUrl;

            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .releaseTime(System.currentTimeMillis())
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .httpUrl(configUrl)
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parseLinks.add(record);
        }
    }

    //得到在售款的url
    private void parseConfigUrl(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"serise_model_price_list\"]/table[1]/tbody/tr").nodes();
        for (int i = 1; i < nodes.size(); i++) {
            String configUrl = nodes.get(i).xpath("./td[5]/p/a[2]/@href").get();
            if (configUrl  == null){
                log.error("There is no link " + crawlerRecord.getHttpRequest().getUrl());
                continue;
            }
            configUrl = XcarUrl + configUrl;
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .releaseTime(System.currentTimeMillis())
                    .httpUrl(configUrl)
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parseLinks.add(record);
        }
    }

    //解析停售款的链接信息
    private void parseTurnPage(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        String url = crawlerRecord.getHttpRequest().getUrl();
        List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"serise_model_price_list\"]/div[1]/div[2]/div/ul/li").nodes();
        String sellout = null;
        for (Selectable node : nodes){
            sellout = node.xpath("./a/text()").get();
            sellout = sellout.substring(0,sellout.length()-1);//得到停售年份
            String[] split = url.split("/");
            //得到停售款
            String ordSeriesUrl = "https://newcar.xcar.com.cn/auto/index.php?r=newcar/SeriseParentIndex/AjaxStopSaleModel&rand=0.5359516740196146&pserid="+split[3]+"&year="+sellout;
            CrawlerRequestRecord record =CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(ordSeriesUrl)
                    .recordKey(ordSeriesUrl)
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .copyBizTags()
                    .copyResultTags()
                    .releaseTime(System.currentTimeMillis())
                    .build();
            Map<String,String> map = new HashMap<>();
            parseLinks.add(record);
        }
    }
    //得到品牌信息
    private void parseBrandMessage(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parseLinks) {
        String url = crawlerRecord.getHttpRequest().getUrl();
        String brandName = page.getHtml().xpath("/html/body/div[@class=\"place_nv\"]/div[@class=\"place\"]/a[2]/text()").get();
        String brandUrl = page.getHtml().xpath("/html/body/div[@class=\"place_nv\"]/div[@class=\"place\"]/a[2]/@href").get();//品牌url
        String[] split = brandUrl.split("/");
        String brandId = split[2];//品牌id
        brandUrl = XcarUrl + brandUrl;

        String motorcycleName = page.getHtml().xpath("/html/body/div[@class=\"place_nv\"]/div[@class=\"place\"]/a[4]/text()").get();
        String motorcycleUrl = page.getHtml().xpath("/html/body/div[@class=\"place_nv\"]/div[@class=\"place\"]/a[4]/@href").get();
        String[] split1 = motorcycleUrl.split("/");
        String motorcycleId = split1[1];
        Map<String,String> map = new HashMap<>();
        map.put("brandName",brandName);
        map.put("brandUrl",brandUrl);
        map.put("brandId",brandId);
        map.put("motorcycleName",motorcycleName);
        map.put("motorcycleUrl",motorcycleUrl);
        map.put("motorcycleId",motorcycleId);
        crawlerRecord.tagsCreator().bizTags().addCustomKV("brandMessage",map);
        crawlerRecord.setNeedWashPage(false);
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerData = new ArrayList<>();
        String url = crawlerRecord.getHttpRequest().getUrl();
        if (crawlerRecord.tagsCreator().resultTags().hasDataType(CrawlerEnum.CrawlerDataType.article)){
            if (url.matches(ConfigUrl)){
                this.washArticle(crawlerRecord, crawlerData, page);
            }
        }


        return crawlerData;
    }

    private void washArticle(CrawlerRequestRecord crawlerRecord, List<CrawlerData> crawlerData, HttpPage page) {
        String year = page.getHtml().xpath("/html/body/div[@class=\"model_wp\"]/div/div[@class=\"new_wt\"]/div[1]/span[@class=\"lt_f1\"]/text()").get();
        String type = page.getHtml().xpath("/html/body/div[@class=\"model_wp\"]/div/div[@class=\"new_wt\"]/div[1]/h1/text()").get();
       // String carMotorcycle = year + type;
        List<Selectable> nodes = page.getHtml().xpath("//*[@id=\"Table1\"]/tbody/tr").nodes();
        List<Map<String,String>> list = new ArrayList<>();
        Map<String,Map<String,String>> maps = new HashMap<>();
        Map<String,String> map = null;
        int a = 1;
        for (int i = 0; i <nodes.size() ; i++) {
            String trId = nodes.get(i).xpath("./@cid").get();
            if (trId != null){
                if (trId.equals(String.valueOf(a))){
                    map = new HashMap<>();
                    ++a;
                    String data = nodes.get(i).xpath("./th[1]//text()").get();
                    maps.put(data,map);
                    continue;
                }
            }

            String td1 = nodes.get(i).xpath("./td[1]/text()").get();
            String td2 = nodes.get(i).xpath("./td[2]/text()").get();
            if (td1 != null || td1 != ""){
                td1 = td1.replaceAll(" ","").replaceAll("\n","");
            }
            if (td2 != null || td2 != ""){
                td2 = td2.replaceAll(" ","").replaceAll("\n","");
            }

            if (td1 == null || td1 == ""){
                td1 = nodes.get(i).xpath("./td[1]/a/text()").get();
            }
            if (td2 == null || td2 == ""){
                td2 = nodes.get(i).xpath("./td[2]/a/text()").get();
            }
            if (td1.equals("外观颜色：") || td1.equals("内饰颜色：")){
                List<Selectable> nodes1 = nodes.get(i).xpath("./td[2]/div").nodes();
                StringBuilder s = new StringBuilder();
                for (Selectable node1 : nodes1){
                    String s1 = node1.xpath("./@title").get();
                    s.append(s1+"/");
                }
                td2 = new String(s);
            }
            td1 = td1.replaceAll(" ","").replaceAll("\n","");
            td2 = td2.replaceAll(" ","").replaceAll("\n","");
            map.put(td1,td2);
        }
        CrawlerData crawlerData1 = CrawlerData.builder()
                .data(crawlerRecord,page)
                .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), CrawlerEnum.CrawlerDataType.article.enumVal(),System.currentTimeMillis()+(new Random().nextInt(10000))))
                .url(crawlerRecord.getHttpRequest().getUrl())
                .releaseTime(System.currentTimeMillis())
                .resultLabelTag(CrawlerEnum.CrawlerDataType.article)
                .flowInPipelineTag("kafka")
                .addContentKV("configMessage", String.valueOf(maps))
                .build();
        crawlerData1.setFilterPipelineResult(true);
        //crawlerData1.tagsCreator().bizTags().addCustomKV("configMessage",list);
        crawlerData.add(crawlerData1);

    }


    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }
    /*
     * 下在失败 重新下载
     * */
    private void recordAgainRequest(CrawlerRequestRecord crawlerRecord, List<CrawlerRequestRecord> linksRecords) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(RECORD_AGAIN_REQUEST)){//判断是否重新下载过
            count = (int)crawlerBusinessTags.getCategoryTag().getKVTag(RECORD_AGAIN_REQUEST).getVal();
            if (count >= 5){
                log.error(DOMAIN + " download page the number of retries exceeds the limit,request url {}",crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }
        String url = crawlerRecord.getHttpRequest().getUrl();
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get(TURN_PAGE_ITEM_REQUEST);
        CrawlerRequestRecord crawlerRequestRecord = null;
        if (type == null){
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .releaseTime(System.currentTimeMillis())
                    .httpUrl(url)
                    .recordKey(crawlerRecord.getRecordKey()+count)
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }else{
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .releaseTime(System.currentTimeMillis())
                    .httpUrl(url)
                    .recordKey(crawlerRecord.getRecordKey()+count)
                    .httpHead("cookie","__jsluid_h=8b69f245411998b6f0ea95ba7c79c40e; __jsl_clearance=1570598566.464|0|h%2FbOP%2BtfLH%2BdlQOX27DEgliP0II%3D; _Xdwuv=5d9d67a86da0f; _Xdwnewuv=1; uv_firstv_refers=http%3A//dealer.xcar.com.cn/d1000/66.htm%3Ftype%3D1%26page%3D1; _PVXuv=5d9d64a4700af; isRemoveCookie=1; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1570598565; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; fw_slc=1%3A1570598564%3B1%3A1570598956%3B1%3A1570598959%3B1%3A1570599006%3B1%3A1570599007; fw_exc=1%3A1570598562%3B1%3A1570598958%3B1%3A1570598960%3B1%3A1570598961%3B1%3A1570599012; fw_pvc=1%3A1570598564%3B1%3A1570598959%3B1%3A1570598961%3B1%3A1570599006%3B1%3A1570599012; fw_clc=1%3A1570598958%3B1%3A1570598960%3B1%3A1570599029%3B1%3A1570599039%3B1%3A1570599042; _Xdwstime=1570599046; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1570599043")
                    .copyBizTags()
                    .copyRequestTags()
                    .build();
        }
        if (crawlerRequestRecord == null){
            return;
        }
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(RECORD_AGAIN_REQUEST,count+1);
        linksRecords.add(crawlerRequestRecord);
    }


    private void cookieUpdate(HttpPage page, CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> parseLinks){
        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());
        HttpPage httpPage = downloader.download(crawlerRequestRecord.getHttpRequest(), crawlerRequestRecord.getHttpConfig());
        String js = httpPage.getRawText().substring(httpPage.getRawText().indexOf("("), httpPage.getRawText().lastIndexOf(")")+1);
        Map<String,String> cookieMap = new HashMap<>();
        try {
            String eval = (String) scriptEngine.eval(js);
            eval =  eval.substring(eval.indexOf("=")+1, eval.indexOf(";"));
            cookieMap.put("__jsl_clearance_s",eval);
            log.info("xcar cookie second update result __jsl_clearance_s={}",eval);
            crawlerRequestRecord.getHttpRequest().addHeader("cookie",processCookie(cookieMap));

            // 2. 生成 第二个 加密 cookie
            httpPage = downloader.download(crawlerRequestRecord.getHttpRequest(), crawlerRequestRecord.getHttpConfig());
            Matcher mtAuthor = Pattern.compile("};go\\((.*?)\\)</script>").matcher(httpPage.getRawText());
            if (mtAuthor.find()){
                String cookieGen = mtAuthor.group(1);
                Json cookieGenJson = new Json(cookieGen);
                String ha = cookieGenJson.jsonPath($_type + ".ha").get();
                log.info("xcar cookie cipher code {}",ha);
                compiledScriptMap.get(ha).eval();
                Invocable invocable = (Invocable) compiledScriptMap.get(ha).getEngine();
                String result = (String) invocable.invokeFunction("go",JSON.parseObject(cookieGen, Map.class));
                result =  result.substring(result.indexOf("=")+1, result.indexOf(";"));
                cookieMap.put("__jsl_clearance_s",result);
                result = processCookie(cookieMap);
                updateLocalAuthInfo(result);
                log.info("xcar cookie second update result {}",result);
            }

        } catch (Exception e) {
            log.error("xcar cookie update error {}",e.getMessage());
        }

    }
    private String processCookie(Map<String,String> cookieMap){
        StringBuffer cookieSB = new StringBuffer();
        for (Map.Entry<String, String> entry : cookieMap.entrySet()) {
            cookieSB.append(entry.getKey())
                    .append("=")
                    .append(entry.getValue())
                    .append(";");
        }
        return cookieSB.substring(0,cookieSB.lastIndexOf(";"));
    }
    private void initCompileScript(){
        if (compiledScriptMap == null){
            compiledScriptMap = new HashMap<>();
        }

        ScriptEngineManager sm = new ScriptEngineManager();
        NashornScriptEngineFactory factory = null;
        for (ScriptEngineFactory f : sm.getEngineFactories()) {
            if (f.getEngineName().equalsIgnoreCase("Oracle Nashorn")) {
                factory = (NashornScriptEngineFactory)f;
                break;
            }
        }
        String[] stringArray = ArrayUtils.toArray("-doe", "--global-per-engine");
        scriptEngine = factory.getScriptEngine(stringArray);
        List<String> ciphers = Arrays.asList("md5", "sha1", "sha256");
        //E:\chance-crawler-development\crawler-dev-bootstrap\src\main\java\com\chance\cc\crawler\development\bootstrap\xcar\js\
       // String filePathPrefix = "E:\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\xcar\\js\\";
        String filePathPrefix = "/data/chance_crawler_runner/domain/xcar/cipher_js/";
        for (String cipher : ciphers) {
            String filePath = filePathPrefix+cipher+".js";
            log.info("xcar cipher js file path {}",filePath);
            final CompiledScript compiled;
            try {
                compiled = ((Compilable)scriptEngine).compile(new FileReader(filePath));
                compiledScriptMap.put(cipher,compiled);
            } catch (ScriptException e) {
                e.printStackTrace();
            } catch (FileNotFoundException e) {
                e.printStackTrace();
            }

        }
    }


}
